#!/usr/bin/env jruby
# -*- coding: us-ascii -*-
# Elizabeth Wiethoff 2008

raise NotImplementedError, 'need version >= 1.8' if RUBY_VERSION < '1.8'
$; = /\s+/m if $DEBUG  # avoid pointless JRuby warning message

require 'find'
require 'yaml'

########################################

def parse_cli
  $full_hash = true
  return Dir.glob('*') if ARGV.empty?
  
  first_arg = ARGV.shift
  if first_arg =~ /^-?-s\w{0,5}/  # -s or --simple
    $full_hash = false
    return Dir.glob('*') if ARGV.empty?
  else
    ARGV.unshift first_arg
  end
  
  return ARGV
end

dirs = parse_cli
require File.join(File.dirname(__FILE__), 'sarraylib')

########################################

yaml_path = File.join(File.dirname(__FILE__), 'superduper.yaml')
yaml = File.open(yaml_path) { |file| YAML.load(file) }

Document.threshold = yaml['threshold']
Document.skip_common_words = yaml['skip_common_words']
path_length_limit = yaml['path_length_limit']
verbosity = yaml['verbose']  # not used
want_ext = yaml['text_extensions'].map {|e| Regexp.new('\.'<< e<< '$')}
prunes = yaml['ignore_regexes'].map {|e| Regexp.new(e)}

########################################

msg = "look for runs of #{Document.threshold} words"
msg += ", skipping common words" if Document.skip_common_words
puts msg

puts "search in #{dirs.inspect}"

puts 'reading files ...'
count = kb = 0 ; pll = path_length_limit
dirs.each do |dir|
  Dir["#{dir}/*"].each do |glob|
    Find.find(glob) do |path|
      Find.prune if File.symlink?(path)
      Find.prune if File.zero?(path) and File.file?(path)
      Find.prune if prunes.any? {|ex| path =~ ex }
      next unless File.file?(path)
      if want_ext.any? {|ex| path =~ ex }
        count += 1 ; kb += File.size(path)/1024
        path_end = path.length < pll ? path : path[-pll, pll]
        puts %Q[#{count} (#{kb}k) "#{path_end}" ...]
        Document.process(path, File.open(path) { |f| f.read })
      end
    end
  end
end

puts 'looking for duplicates ...'
dupes = Dupes.find_em

puts 'tossing garbage ...'
toss_garbage

puts ; puts 'FOUND THESE DUPLICATES:'
dupes.sort.print path_length_limit
puts ; puts 'FINIS!'
